# coding: utf-8
import pandas as pd
import pytz
from datetime import datetime
import pymongo
from time import time
import os
import timeUtils
import logging

"""
把历史行情数据导入到mongodb
"""
logger = logging.basicConfig(level=logging.INFO)
t0 = time()
path = "/home/hanyu/tonglian/oneMinData/futures/allfutures_2007-01-01_to_2015-11-17/"
path2 = "/home/hanyu/tonglian/oneMinData/futures/allfutures_2007-01-01_to_2015-11-17/"
pathWin = "D:\\wks_python\\allfutures_data\\zn\\"  # allfutures_2015-11-17_to_2015-12-2
pwd = os.getcwd()
filelist = os.listdir(pathWin)
column_names = {'ticker': 'symbol', 'closePrice': 'close', 'openPrice': 'open', 'highPrice': 'high', 'lowPrice': 'low',
                'totalVolume': 'volume'}

client = pymongo.MongoClient('121.40.212.219', 27017)  # '121.40.212.219', 27017
db = client.future_data
# fi = open(pwd + '/log_min_data.txt', 'a')
for filename in filelist:
    logging.info(filename)
    # fi.write(filename + '\n')
    df = pd.read_csv(pathWin + filename)
    symbol = filename.split(".")[0]
    alpha = filter(str.isalpha, symbol)
    num = filter(str.isdigit, symbol)
    if len(num) < 4:
        if int(num[0]) > 7:
            num = '0' + num
        elif int(num[0]) < 7:
            num = '1' + num
    symbol = alpha + num
    if df.empty:
        pass
    df = df[df['barTime'] != '99:99']

    df.rename(columns=column_names, inplace=True)
    df = df[df['close'] != 0]
    df['dateTime'] = df['dataDate'] + ' ' + df['barTime'] + ":00"
    tz = pytz.timezone(pytz.country_timezones('cn')[0])
    df['dateTime'] = df['dateTime'].apply(lambda x: tz.localize(datetime.strptime(x, '%Y-%m-%d %H:%M:%S')))

    db.one_min_data.insert_many(df.reset_index().to_dict(orient='records'))
    logging.info("-------one min done-------")
    df_m60 = df.copy()
    df = df.set_index('dateTime')

    # ---------------write in 5 minute data----------------
    df_m5 = pd.DataFrame([])
    df_m5['open'] = df['open'].resample('5min', how='first', closed='right', label='right').dropna()
    df_m5['close'] = df['close'].resample('5min', how='last', closed='right', label='right').dropna()
    df_m5['high'] = df['high'].resample('5min', how='max', closed='right', label='right').dropna()
    df_m5['low'] = df['low'].resample('5min', how='min', closed='right', label='right').dropna()
    # df_m5['dateTime'] = df['dateTime'].resample('5min', how='last', closed='right', label='right').dropna()
    df_m5['totalValue'] = df['totalValue'].resample('5min', how='sum', closed='right', label='right').dropna()
    df_m5['volume'] = df['volume'].resample('5min', how='sum', closed='right', label='right').dropna()
    df_m5['openInterest'] = df['openInterest'].resample('5min', how='last', closed='right', label='right').dropna()
    df_m5['symbol'] = symbol
    db.min_5_data.insert_many(df_m5.reset_index().to_dict(orient='records'))
    logging.info("-------5 min done-------")
    # ---------------write in 15 minute data----------------
    df_m15 = pd.DataFrame([])
    df_m15['open'] = df_m5['open'].resample('15min', how='first', closed='right', label='right').dropna()
    df_m15['close'] = df_m5['close'].resample('15min', how='last', closed='right', label='right').dropna()
    df_m15['high'] = df_m5['high'].resample('15min', how='max', closed='right', label='right').dropna()
    df_m15['low'] = df_m5['low'].resample('15min', how='min', closed='right', label='right').dropna()
    df_m15['totalValue'] = df_m5['totalValue'].resample('5min', how='sum', closed='right', label='right').dropna()
    df_m15['volume'] = df_m5['volume'].resample('5min', how='sum', closed='right', label='right').dropna()
    df_m15['openInterest'] = df_m5['openInterest'].resample('5min', how='last', closed='right', label='right').dropna()
    df_m15['symbol'] = symbol
    db.min_15_data.insert_many(df_m15.reset_index().to_dict(orient='records'))
    logging.info("-------15 min done-------")
    # ---------------write in 60 minute data----------------
    df_m60['dateTime'] = timeUtils.dfMinute60(df_m60['dateTime'])
    df_m60['symbol'] = symbol
    grouped = df_m60.groupby('dateTime')
    functions = {'open': 'first', 'close': 'last', 'high': 'max', 'low': 'min', 'volume': 'sum', 'totalValue': 'sum',
                 'openInterest': 'last'}
    rsDF = grouped.agg(functions)
    rsDF['symbol'] = symbol
    db.min_60_data.insert_many(rsDF.reset_index().to_dict(orient='records'))
    logging.info("-------60 min done-------")
    # ---------------write in day data----------------
    df_day = pd.DataFrame([])
    df_day['open'] = rsDF['open'].resample('B', how='first', closed='right', label='right').dropna()
    df_day['close'] = rsDF['close'].resample('B', how='last', closed='right', label='right').dropna()
    df_day['high'] = rsDF['high'].resample('B', how='max', closed='right', label='right').dropna()
    df_day['low'] = rsDF['low'].resample('B', how='min', closed='right', label='right').dropna()
    df_day['totalValue'] = rsDF['totalValue'].resample('B', how='sum', closed='right', label='right').dropna()
    df_day['volume'] = rsDF['volume'].resample('B', how='sum', closed='right', label='right').dropna()
    df_day['openInterest'] = rsDF['openInterest'].resample('B', how='last', closed='right', label='right').dropna()
    df_day['symbol'] = symbol
    db.day.insert_many(df_day.reset_index().to_dict(orient='records'))
    logging.info("-------day done-------")

# gtm = df['open'].resample('dateTime', how='first')

# if the first row of pChange is NaN, then we ignore the first row
#     d = tz.localize(datetime(2015, 9, 30, 0, 0))
#     if math.isnan(df.pChange.iloc[0]) and df.dateTime.iloc[0] == d:
#         if df[1:].empty:
#             pass
#         else:
#             db.original_prices.insert_many(df[1:].to_dict(orient='records'))
#     else:
#         db.original_prices.insert_many(df.to_dict(orient='records'))
#     fi.write(filename + '\n')
#
# fi.close()
# print('finished in %.2fs' % (time() - t0))
